In [1]:
import pickle
In [3]:
with open('result_list.pickle','rb') as f:
data = pickle.load(f)
In [4]:
clean_d=[]
for i in data:
clean_d.append(i.replace('Within Set Sum of Squared Error = ','').replace('for k =','').split())
In [5]:
from matplotlib import pyplot as plt
In [6]:
x=[]
y=[]
for i in clean_d:
x.append(int(i[1]))
y.append(float(i[0]))
In [7]:
min(x)
Out[7]:
In [10]:
fig, ax = plt.subplots()
ax.grid(True)
ax.plot(x,y)
plt.axis([min(x),max(x),min(y),max(y)])
plt.show()
In [9]:
from pyspark.mllib.clustering import KMeans, KMeansModel
In [11]:
sameModel = KMeansModel.load(sc, "hdfs:///user/cluster_model/cluster_model_k_17")
In [13]:
with open('success_vec_protocol_1.pickle','rb') as objectf:
data = pickle.load(objectf)
In [14]:
keys_rdd = sc.parallelize(list(data.keys()))
In [15]:
key_value_rdd = keys_rdd.map(lambda x: (x,data[x].reshape((4096,))))
In [16]:
array_rdd = key_value_rdd.map(lambda (k,v):v)
In [17]:
from scipy.spatial.distance import euclidean
In [18]:
distance = array_rdd.map(lambda point :(sameModel.predict(point),euclidean(point,sameModel.centers[sameModel.predict(point)])))
In [19]:
radius = distance.reduceByKey(max)
In [20]:
radius.collect()
Out[20]:
In [21]:
rd_hash = array_rdd.map(lambda point :(sameModel.predict(point),point))
In [22]:
rd_group = rd_hash.groupByKey()
In [23]:
group = rd_group.collect()
In [24]:
for i in group:
print (i[0],len([j for j in i[1]]))
In [47]:
for i in group:
if i[0]==16:
d =[j for j in i[1]]
In [48]:
for idx,val in enumerate(d):
d[idx]=val.reshape((1,4096))
In [49]:
import random
In [51]:
test_d = d
In [200]:
#test_d = d
In [52]:
reverse_hash = key_value_rdd.map(lambda(k,v):(k,v.reshape((1,4096))))
In [53]:
t = reverse_hash.take(1)
In [54]:
t[0][1].shape
Out[54]:
In [55]:
d[0].shape
Out[55]:
In [56]:
import numpy as np
In [57]:
brod = sc.broadcast(test_d)
In [58]:
def fil(k,v):
for i in brod.value:
if np.array_equal(i,v):
return k
In [59]:
subset = reverse_hash.filter(lambda (k,v):fil(k,v)).collect()
In [60]:
len(subset)
Out[60]:
In [41]:
subset[0][0]
Out[41]:
In [42]:
img_dir = '/mnt/homes_img'
In [ ]:
In [43]:
%matplotlib inline
import matplotlib.pyplot as plt
from scipy.ndimage import imread
In [61]:
for i in subset:
test_img = imread(img_dir+"/"+i[0])
print (i[0])
plt.imshow(test_img)
plt.show()
In [ ]: